package model;


import inference.EntityFactor;
import inference.PredicateFactor;
import inference.RootFactor;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.SortedSet;
import java.util.TreeSet;

import com.analog.lyric.dimple.factorfunctions.core.FactorTable;
import com.analog.lyric.dimple.factorfunctions.core.IFactorTable;
import com.analog.lyric.dimple.model.Discrete;
import com.analog.lyric.dimple.model.DiscreteDomain;

import utils.Date;
import utils.IntPair;
import utils.Pair;
import utils.PrettyPrinter;
import utils.TextUtils;


public class ModelParameters {
	private int num_types_per_column = 0;
	private int num_possible_table_topics = 10;
	private int max_num_hidden_columns = 1;
	private int max_columns = 1;
	
	private double new_predicate_weight = 0.001;
	private double new_type_weight = 0.00001;
	private double new_entity_weight = 0.00000001;
	private double prob_of_wrong_functional_rel = 0.000001;
	private double prob_of_new_obj_in_nonfunc_rel = 0.000001;
	private double laplace_smoothing_alpha = 0.01;
	private double approximate_type_domain_size = 1000;
	
	private boolean ignore_table_topic = false;
	private boolean use_unknown_topic = false;
	private static boolean VERBOSE = true;
	
	private TableInformation table_info;
	private ArrayList<HashSet<Integer> > possible_visible_column_types;
	private HashSet<Integer> possible_hidden_column_types;
	private ArrayList<HashSet<Integer> > possible_hidden_column_types_by_parent;
	private HashMap<IntPair, Integer> adj_col_pairs;
	private HashMap<IntPair, HashSet<Integer> > col_to_acceptable_preds;
	
	
	private Discrete[][] entity_variables;
	private Discrete[] predicate_variables;
	private Discrete[] column_type_variables;
	private Discrete[] topic_variables;
	private Discrete[] topic_type_variables;
	
	private IFactorTable[][][] entity_factor_cache;
	private IFactorTable[][] predicate_factor_cache;
	private IFactorTable[] topic_factor_cache;
	
	public ModelParameters(TableInformation table_info) {
		this(table_info, 0);
	}
	
	public ModelParameters(TableInformation table_info, int num_types_per_column) {
		this.table_info = table_info;
		this.num_types_per_column = num_types_per_column; 
		possible_visible_column_types = new ArrayList<HashSet<Integer> >(table_info.numVisibleCols());
		for (int c = 0; c < table_info.numVisibleCols(); c++)
			possible_visible_column_types.add(new HashSet<Integer>());
		possible_hidden_column_types = new HashSet<Integer>();
		possible_hidden_column_types_by_parent = new ArrayList<HashSet<Integer> >();
		adj_col_pairs = new HashMap<IntPair, Integer>();
		max_columns = table_info.numVisibleCols() + max_num_hidden_columns;
		approximate_type_domain_size = table_info.numRows() * 100;
	}
	
	public TableInformation getTableInformation() {
		return table_info;
	}
	
	public static void setVerbose(boolean verbose) {
		VERBOSE = verbose;
	}
	
	public HashSet<Integer> getPossibleTypesForColumn(int col) {
		if (col < 0)
			return null;
		if (col < table_info.numVisibleCols())
			return possible_visible_column_types.get(col);
		if (col < max_columns)
			return possible_hidden_column_types;
		return null;
	}
	
	public void setNumTypesPerColumn(int num_types_per_column) throws Exception {
		if (num_types_per_column < 0) {
			System.err.println("Number of types per column must be a positive integer" +
							   "or 0 (if the user wants all types to be considered.");
			throw new Exception();
		}
			
		this.num_types_per_column = num_types_per_column;
	}
	
	public void setNumPossibleTableTopics(int num_possible_table_topics) throws Exception {
		if (num_possible_table_topics < 0) {
			System.err.println("Number of possible table topics must be a positive integer" +
							   "or 0 (if the user wants all topics to be considered.");
			throw new Exception();
		}
			
		this.num_possible_table_topics = num_possible_table_topics;
	}
	
	public void setMaxNumHiddenColumns(int max_num_hidden_columns) throws Exception {
		if (max_num_hidden_columns < 0) {
			System.err.println("Number of hidden columns cannot be a negative number.");
			throw new Exception();
		}
			
		this.max_num_hidden_columns = max_num_hidden_columns;
		max_columns = table_info.numVisibleCols() + max_num_hidden_columns;
	}
	
	public int getMaxNumHiddenColumns() {
		return max_num_hidden_columns;
	}
	
	public int getMaxColumns() {
		return max_columns;
	}
	
	public boolean useUnknownTopic() {
		return use_unknown_topic;
	}
	
	public boolean ignoreTableTopic() {
		return ignore_table_topic;
	}
	
	public Discrete getColumnTypeVariable(int col) {
		if (col >= column_type_variables.length || col < 0)
			return null;
		return column_type_variables[col];
	}
	
	public Discrete getPredicateVariable(int col) {
		if (col >= predicate_variables.length || col < 0)
			return null;
		return predicate_variables[col];
	}
	
	public Discrete getTopicVariable(int root_col) {
		if (root_col >= topic_variables.length || root_col < 0)
			return null;
		return topic_variables[root_col];
	}
	
	public Discrete getTopicTypeVariable(int root_col) {
		if (root_col >= topic_type_variables.length || root_col < 0)
			return null;
		return topic_type_variables[root_col];
	}
	
	public Discrete getEntityVariable(int row, int col) {
		return entity_variables[col][row];
	}
	
	private void collectPossibleTypesForVisibleColumns() {
		if (num_types_per_column == 0)
			collectAllPossibleTypesForVisibleColumns();
		else
			collectNPossibleTypesForVisibleColumns(num_types_per_column);
	}
	
//	private void collectPossibleTypesForVisibleColumns(int num_types_per_column) {
//		collectNPossibleTypesForVisibleColumns(num_types_per_column);
//	}
	
	private void collectPossibleCvtTypesForHiddenColumn() {
		possible_hidden_column_types.clear();
		if (max_columns > table_info.numVisibleCols())  {
			int col = table_info.numVisibleCols();
			for (int row = 0; row < table_info.numRows(); row++) {
				Object[] entities = entity_variables[col][row].getDomain().getElements();
				for (int ent = 0; ent < entities.length - 1; ent++) {  // Last entity is the UNK.
					Integer ent_id = (Integer)(entities[ent]);
					HashSet<Integer> type_ids = table_info.getTypesForEntityId(ent_id);
					if (type_ids != null) {
						possible_hidden_column_types.addAll(type_ids);
					}
				}
			}
		}
	}
	
	private void collectNPossibleTypesForVisibleColumns(int num_types_per_column) {
		int float_id = table_info.getOrAddIdForString("/type/float");		
		int int_id = table_info.getOrAddIdForString("/type/int");
		int date_id = table_info.getOrAddIdForString("/type/datetime");
		for (int col = 0; col < table_info.numVisibleCols(); col++) {
			HashSet<Integer> column_types = possible_visible_column_types.get(col);
			column_types.clear();
			HashMap<Integer, Integer> column_types_to_cnts = new HashMap<Integer, Integer>();
			for (int row = 0; row < table_info.numRows(); row++) {
				Integer[] cell_entity_ids = table_info.getCellEntities(row, col);
				if (cell_entity_ids != null) {
					for (int ent = 0; ent < cell_entity_ids.length; ent++) {
						Integer entity_id = cell_entity_ids[ent];
						String entity = table_info.getStringForId(entity_id);
						if (TextUtils.isMid(entity)) {
							HashSet<Integer> entity_types = table_info.getTypesForEntity(entity);
							if (entity_types == null)
								continue;
							for (Integer type_id : entity_types) {
								Integer cnt = column_types_to_cnts.get(type_id);
								if (cnt == null) {
									column_types_to_cnts.put(type_id, 1);
								} else {
									column_types_to_cnts.put(type_id, cnt.intValue() + 1);
								}
							}
						} else {
							Double double_value = TextUtils.parseNumericString(entity);
							if (double_value != null) {
								Integer cnt = column_types_to_cnts.get(float_id);
								if (cnt == null) {
									column_types_to_cnts.put(float_id, 1);
								} else {
									column_types_to_cnts.put(float_id, cnt.intValue() + 1);
								}
								table_info.addTypeIdForEntity(entity, float_id);
								Integer int_value = TextUtils.parseIntegerString(entity);
								System.out.println(entity + " " + int_value);
								if (int_value != null) {
									cnt = column_types_to_cnts.get(int_id);
									if (cnt == null) {
										column_types_to_cnts.put(int_id, 1);
									} else {
										column_types_to_cnts.put(int_id, cnt.intValue() + 1);
									}
									table_info.addTypeIdForEntity(entity, int_id);
									if (int_value.intValue() > 1000) {
										cnt = column_types_to_cnts.get(date_id);
										if (cnt == null) {
											column_types_to_cnts.put(date_id, 1);
										} else {
											column_types_to_cnts.put(date_id, cnt.intValue() + 1);
										}
										table_info.addTypeIdForEntity(entity, date_id);
									}
								}
							} else {
								Date date_value = Date.parseTextDate(entity);
								if (date_value != null) {
									Integer cnt = column_types_to_cnts.get(date_id);
									if (cnt == null) {
										column_types_to_cnts.put(date_id, 1);
									} else {
										column_types_to_cnts.put(date_id, cnt.intValue() + 1);
									}
									table_info.addTypeIdForEntity(entity, date_id);
								}
							}
						}
					}
				}
			}
			
			SortedSet<Map.Entry<Integer, Integer> > sorted_types = new TreeSet<Map.Entry<Integer, Integer> >(
					new Comparator<Map.Entry<Integer, Integer> >() {
				public int compare(Map.Entry<Integer, Integer> e1, Map.Entry<Integer, Integer> e2) {
					return e2.getValue().compareTo(e1.getValue());
				}
			});
			sorted_types.addAll(column_types_to_cnts.entrySet());
			int max_types = (num_types_per_column < sorted_types.size()) ? num_types_per_column : sorted_types.size();
			Iterator<Map.Entry<Integer, Integer> > it = sorted_types.iterator();
			if (VERBOSE)
				System.out.print(col + ":  ");
			for (int i = 0; i < max_types; i++) {
				Integer type = it.next().getKey();
				if (VERBOSE)
					System.out.print(table_info.getStringForId(type) + ":" + column_types_to_cnts.get(type) + ", ");
				column_types.add(type);
			}
			if (VERBOSE)
				System.out.println();
		}
		
		
	}
	
	private void collectAllPossibleTypesForVisibleColumns() {
		int float_id = table_info.getIdForString("/type/float");		
		int int_id = table_info.getIdForString("/type/int");
		int date_id = table_info.getIdForString("/type/datetime");
		for (int col = 0; col < table_info.numVisibleCols(); col++) {
			HashSet<Integer> column_types = possible_visible_column_types.get(col);
			column_types.clear();
			for (int row = 0; row < table_info.numRows(); row++) {
				Integer[] cell_entity_ids = table_info.getCellEntities(row, col);
				if (cell_entity_ids != null) {
					for (int ent = 0; ent < cell_entity_ids.length; ent++) {
						Integer entity_id = cell_entity_ids[ent];
						String entity = table_info.getStringForId(entity_id);
						if (TextUtils.isMid(entity)) {
							HashSet<Integer> entity_types = table_info.getTypesForEntity(entity);
							if (entity_types != null)
								column_types.addAll(entity_types);
						} else {
							Double double_value = TextUtils.parseNumericString(entity);
							if (double_value != null) {
								column_types.add(float_id);
								table_info.addTypeIdForEntity(entity, float_id);
								Integer int_value = TextUtils.parseIntegerString(entity);
								if (int_value != null) {
									column_types.add(int_id);
									table_info.addTypeIdForEntity(entity, int_id);
									if (int_value.intValue() > 1000) {
										column_types.add(date_id);
										table_info.addTypeIdForEntity(entity, date_id);
									}
								}
							} else {
								Date date_value = Date.parseDateString(entity);
								if (date_value != null) {
									column_types.add(date_id);
									table_info.addTypeIdForEntity(entity, date_id);
								}
							}
						}
					}
				}
			}
			if (VERBOSE) {
				System.out.print(col + ":  ");
				Iterator<Integer> it = column_types.iterator();
				while (it.hasNext()) {
					Integer type = it.next();
					System.out.print(table_info.getStringForId(type) + ", ");
					column_types.add(type);
				}
				System.out.println();
			}
		}
	}
	
	public void AddRawStringTypesToModel() {
		int text_id = table_info.getOrAddIdForString("/type/text");
		int rawstring_id = table_info.getOrAddIdForString("/type/rawstring");
		String[][] table_text = table_info.getCellText();
		for (int col = 0; col < table_info.numVisibleCols(); col++) {
			HashSet<Integer> column_types = possible_visible_column_types.get(col);
			for (int row = 0; row < table_info.numRows(); row++) {
				String cell_text = table_text[col][row];
				if (cell_text.equals("")) continue;
				if (TextUtils.parseNumericString(cell_text) != null) continue;
				if (Date.parseTextDate(cell_text) != null) continue;
				Integer ent_id = table_info.getOrAddIdForString(cell_text);
				table_info.addTypeIdForEntityId(ent_id, text_id);
				table_info.addTypeIdForEntityId(ent_id, rawstring_id);
				column_types.add(text_id);
				column_types.add(rawstring_id);
			}
		}
	}
	
	public void createVariablesForModel() {
		collectPossibleTypesForVisibleColumns();
		AddRawStringTypesToModel();
		if (max_num_hidden_columns == 0)
			createVariablesForAllVisibleModel(null);
		else
			createVariablesForModelWithPossibleHiddenCols(null);
	}
	
	public void createVariablesForModel(HashSet<Integer> fixed_table_topic_ids) {
		collectPossibleTypesForVisibleColumns();
		if (max_num_hidden_columns == 0)
			createVariablesForAllVisibleModel(fixed_table_topic_ids);
		else
			createVariablesForModelWithPossibleHiddenCols(fixed_table_topic_ids);
	}
	
	private void createVariablesForModelWithPossibleHiddenCols(
			HashSet<Integer> fixed_table_topic_ids) {
		int max_cols = table_info.numVisibleCols() + max_num_hidden_columns;
		column_type_variables = new Discrete[max_cols];
		predicate_variables = new Discrete[max_cols];
		entity_variables = new Discrete[max_cols][table_info.numRows()];
		topic_variables = new Discrete[max_cols];
		topic_type_variables = new Discrete[max_cols];
		
		ArrayList<HashMap<Integer, Integer> > root_predicates_arr = null;
		if (!ignore_table_topic) {
			if (fixed_table_topic_ids == null || fixed_table_topic_ids.size() == 0) {
				root_predicates_arr = initializeTableTopicVariablesForVisibleRootColumns();
			} else {
				initializeTableTopicVariables(fixed_table_topic_ids);
			}
		}
		initializeEntityVariables();
		collectPossibleCvtTypesForHiddenColumn();
		createBackboneAdjacencyMatrix();
		HashMap<Integer, Integer> root_preds = initializeTableTopicVariablesForHiddenRootColumns(entity_variables[table_info.numVisibleCols()]);
		root_predicates_arr.add(root_preds);
		initializePredicateAndColumnTypeVariables(fixed_table_topic_ids, root_predicates_arr, null);
	}
	
	private void createVariablesForAllVisibleModel(HashSet<Integer> fixed_table_topic_ids) {
		column_type_variables = new Discrete[table_info.numVisibleCols()];
		predicate_variables = new Discrete[table_info.numVisibleCols()];
		entity_variables = new Discrete[table_info.numVisibleCols()][table_info.numRows()];
		topic_variables = new Discrete[table_info.numVisibleCols()];
		
		if (!ignore_table_topic) {
			if (fixed_table_topic_ids == null || fixed_table_topic_ids.size() == 0) {
				ArrayList<HashMap<Integer, Integer> > root_predicates_arr = initializeTableTopicVariablesForVisibleRootColumns();
				initializeEntityVariables();
				createBackboneAdjacencyMatrix();
				initializePredicateAndColumnTypeVariables(fixed_table_topic_ids, root_predicates_arr, null);
			} else {
				initializeTableTopicVariables(fixed_table_topic_ids);
				initializeEntityVariables();
				createBackboneAdjacencyMatrix();
				initializePredicateAndColumnTypeVariables(fixed_table_topic_ids, null, null);
			}
		} else {
			initializeEntityVariables();
			createBackboneAdjacencyMatrix();
			initializePredicateAndColumnTypeVariables(fixed_table_topic_ids, null, null);
		}
	}
	
	private void initializeTableTopicVariables(HashSet<Integer> fixed_table_topic_ids) {
		HashSet<Integer> root_types = new HashSet<Integer>();
		Object[] domain_arr = fixed_table_topic_ids.toArray();
		double[] domain_prob = new double[domain_arr.length];
		double prob = 1.0 / domain_arr.length;
		for (int i = 0; i < domain_arr.length; i++) {
			domain_prob[i] = prob;
			root_types.addAll(table_info.getTypesForEntityId((Integer)(domain_arr[i])));
		}
		Object[] type_arr = root_types.toArray();
		double[] type_prob = new double[type_arr.length];
		prob = 1.0 / type_prob.length;
		for (int i = 0; i < type_prob.length; i++) {
			type_prob[i] = prob;
		}
		int max_cols = table_info.numVisibleCols() + max_num_hidden_columns;
		for (int col = 0; col < max_cols; col++) {
			topic_variables[col] = createDiscreteVariable(domain_arr, domain_prob, "TableTopic");
			topic_type_variables[col] = createDiscreteVariable(type_arr, type_prob, "TableTopicType");
		}
	}
	
	private ArrayList<HashMap<Integer, Integer> > initializeTableTopicVariablesForVisibleRootColumns() {
		ArrayList<HashMap<Integer, Integer> > root_predicates_arr = new ArrayList<HashMap<Integer, Integer> >();
		Integer unk_topic_type_id = table_info.getOrAddIdForString("UNKTOPICTYPE");
		for (int col = 0; col < table_info.numVisibleCols(); col++) {
			HashMap<Integer, Integer> root_predicates = new HashMap<Integer, Integer>();
			HashSet<Integer> root_types = new HashSet<Integer>();
			SortedSet<Map.Entry<Pair<Integer, Integer>, Integer> > sorted_topic_counts = 
					table_info.findPossibleTableTopics(col);
			
			Object[] domain_arr;
			double[] domain_prob;
			if (sorted_topic_counts.size() == 0) {
				Integer unk_topic_id = table_info.getOrAddIdForString("UNKTOPIC");
				domain_arr = new Integer[]{unk_topic_id};
				domain_prob = new double[]{1.0};
			} else if (use_unknown_topic) {
				Integer unk_topic_id = table_info.getOrAddIdForString("UNKTOPIC");
				int num_topics_used = (num_possible_table_topics == 0 ?
						sorted_topic_counts.size() : Math.min(sorted_topic_counts.size(), num_possible_table_topics)) + 1;
				int total_counts = 1;
				Iterator<Map.Entry<Pair<Integer, Integer>, Integer> > it = sorted_topic_counts.iterator();
				domain_arr = new Object[num_topics_used];
				domain_prob = new double[num_topics_used];
				for (int i = 0; i < num_topics_used - 1; i++) {
					Map.Entry<Pair<Integer, Integer>, Integer> entry = it.next();
					Integer reverse_id = table_info.getOrMakeReversePredicateId(entry.getKey().getFirst());
					Integer cnt = root_predicates.get(reverse_id);
					if (cnt == null)
						root_predicates.put(reverse_id, 1);
					else
						root_predicates.put(reverse_id, cnt.intValue() + 1);
					domain_arr[i] = entry.getKey().getSecond();
					root_types.addAll(table_info.getTypesForEntityId(entry.getKey().getSecond()));
					
					total_counts += entry.getValue();
					domain_prob[i] = entry.getValue();
				}
				domain_arr[num_topics_used - 1] = unk_topic_id;
				domain_prob[num_topics_used - 1] = 1;
				for (int i = 0; i < num_topics_used; i++) {
					domain_prob[i] /= total_counts;
				}
				IntPair col_pair = new IntPair(-1, col);
				adj_col_pairs.put(col_pair, total_counts - 1);
			} else {
				int num_topics_used = (num_possible_table_topics == 0 ?
						sorted_topic_counts.size() : Math.min(sorted_topic_counts.size(), num_possible_table_topics));
				int total_counts = 0;
				Iterator<Map.Entry<Pair<Integer, Integer>, Integer> > it = sorted_topic_counts.iterator();
				domain_arr = new Object[num_topics_used];
				domain_prob = new double[num_topics_used];
				for (int i = 0; i < num_topics_used; i++) {
					Map.Entry<Pair<Integer, Integer>, Integer> entry = it.next();
					Integer reverse_id = table_info.getOrMakeReversePredicateId(entry.getKey().getFirst());
					Integer cnt = root_predicates.get(reverse_id);
					if (cnt == null)
						root_predicates.put(reverse_id, 1);
					else
						root_predicates.put(reverse_id, cnt.intValue() + 1);
					domain_arr[i] = entry.getKey().getSecond();
					root_types.addAll(table_info.getTypesForEntityId(entry.getKey().getSecond()));
					total_counts += entry.getValue();
					domain_prob[i] = entry.getValue();
				}
				for (int i = 0; i < num_topics_used; i++) {
					domain_prob[i] /= total_counts;
				}
				IntPair col_pair = new IntPair(-1, col);
				adj_col_pairs.put(col_pair, total_counts);
			}
			topic_variables[col] = createDiscreteVariable(domain_arr, domain_prob, "TableTopic");
			Object[] type_arr = new Object[root_types.size() + 1];
			double[] type_prob = new double[root_types.size() + 1];
			double unk_type_prob = (1.0 / (root_types.size())) * new_type_weight;
			double non_unk_type_prob = ((1.0 - unk_type_prob) / (root_types.size()));
			if (root_types.size() == 0) {
				unk_type_prob = 1.0;
			}
			int t = 0;
			for (Integer type : root_types) {
				type_arr[t] = type;
				type_prob[t] = non_unk_type_prob;
				t++;
			}
			type_arr[root_types.size()] = unk_topic_type_id;
			type_prob[root_types.size()] = unk_type_prob;
			topic_type_variables[col] = createDiscreteVariable(type_arr, type_prob, "TableTopicType");
			root_predicates_arr.add(root_predicates);
		}
		return root_predicates_arr;
	}
	
	private HashMap<Integer, Integer> initializeTableTopicVariablesForHiddenRootColumns(Discrete[] column_variables) {
		if (max_columns == table_info.numVisibleCols())
			return null;
		int col = table_info.numVisibleCols();
		HashMap<Integer, Integer> root_predicates = new HashMap<Integer, Integer>();
		HashSet<Integer> root_types = new HashSet<Integer>();
		Integer unk_topic_type_id = table_info.getOrAddIdForString("UNKTOPICTYPE");
		SortedSet<Map.Entry<Pair<Integer, Integer>, Integer> > sorted_topic_counts = 
				table_info.findPossibleTableTopics(column_variables);
		if (VERBOSE) {
			for (Map.Entry<Pair<Integer, Integer>, Integer> count : sorted_topic_counts) {
				System.out.println(table_info.getStringForId(count.getKey().getFirst()) + " " + table_info.getStringForId(count.getKey().getSecond()) + "  " + count.getValue()); 
			}
			System.out.println(sorted_topic_counts.size());
		}
		
		Object[] domain_arr;
		double[] domain_prob;
		if (sorted_topic_counts.size() == 0) {
			Integer unk_topic_id = table_info.getOrAddIdForString("UNKTOPIC");
			domain_arr = new Integer[]{unk_topic_id};
			domain_prob = new double[]{1.0};
		} else if (use_unknown_topic) {
			Integer unk_topic_id = table_info.getOrAddIdForString("UNKTOPIC");
			int num_topics_used = (num_possible_table_topics == 0 ?
					sorted_topic_counts.size() : Math.min(sorted_topic_counts.size(), num_possible_table_topics)) + 1;
			int total_counts = 1;
			Iterator<Map.Entry<Pair<Integer, Integer>, Integer> > it = sorted_topic_counts.iterator();
			domain_arr = new Object[num_topics_used];
			domain_prob = new double[num_topics_used];
			for (int i = 0; i < num_topics_used - 1; i++) {
				Map.Entry<Pair<Integer, Integer>, Integer> entry = it.next();
				Integer reverse_id = table_info.getOrMakeReversePredicateId(entry.getKey().getFirst());
				Integer cnt = root_predicates.get(reverse_id);
				if (cnt == null)
					root_predicates.put(reverse_id, 1);
				else
					root_predicates.put(reverse_id, cnt.intValue() + 1);
				domain_arr[i] = entry.getKey().getSecond();
				root_types.addAll(table_info.getTypesForEntityId(entry.getKey().getSecond()));
				total_counts += entry.getValue();
				domain_prob[i] = entry.getValue();
			}
			domain_arr[num_topics_used - 1] = unk_topic_id;
			domain_prob[num_topics_used - 1] = 1;
			if (total_counts != 0) {
				for (int i = 0; i < num_topics_used; i++) {
					domain_prob[i] /= (double)total_counts;
				}
			}
			IntPair col_pair = new IntPair(-1, col);
			adj_col_pairs.put(col_pair, total_counts - 1);
		} else {
			int num_topics_used = (num_possible_table_topics == 0 ?
					sorted_topic_counts.size() : Math.min(sorted_topic_counts.size(), num_possible_table_topics));
			int total_counts = 0;
			Iterator<Map.Entry<Pair<Integer, Integer>, Integer> > it = sorted_topic_counts.iterator();
			domain_arr = new Object[num_topics_used];
			domain_prob = new double[num_topics_used];
			for (int i = 0; i < num_topics_used; i++) {
				Map.Entry<Pair<Integer, Integer>, Integer> entry = it.next();
				Integer reverse_id = table_info.getOrMakeReversePredicateId(entry.getKey().getFirst());
				Integer cnt = root_predicates.get(reverse_id);
				if (cnt == null)
					root_predicates.put(reverse_id, 1);
				else
					root_predicates.put(reverse_id, cnt.intValue() + 1);
				domain_arr[i] = entry.getKey().getSecond();
				root_types.addAll(table_info.getTypesForEntityId(entry.getKey().getSecond()));
				total_counts += entry.getValue();
				domain_prob[i] = entry.getValue();
			}
			for (int i = 0; i < num_topics_used; i++) {
				domain_prob[i] /= total_counts;
			}
			IntPair col_pair = new IntPair(-1, col);
			adj_col_pairs.put(col_pair, total_counts);
		}
		Object[] type_arr = new Object[root_types.size() + 1];
		double[] type_prob = new double[root_types.size() + 1];
		double unk_type_prob = (1.0 / (root_types.size())) * new_type_weight;
		double non_unk_type_prob = ((1.0 - unk_type_prob) / (root_types.size()));
		if (root_types.size() == 0) {
			unk_type_prob = 1.0;
		}
		int t = 0;
		for (Integer type : root_types) {
			type_arr[t] = type;
			type_prob[t] = non_unk_type_prob;
			t++;
		}
		type_arr[root_types.size()] = unk_topic_type_id;
		type_prob[root_types.size()] = unk_type_prob;
		topic_type_variables[col] = createDiscreteVariable(type_arr, type_prob, "TableTopicType");
		for (col = table_info.numVisibleCols(); col < max_columns; col++) {
			topic_variables[col] = createDiscreteVariable(domain_arr, domain_prob, "TableTopic");
		}
		return root_predicates;
	}
	
	private void initializeEntityVariables() {
		Integer unkent_id = table_info.getOrAddIdForString("UNKENT");
		for (int col = 0; col < table_info.numVisibleCols(); col++) {
			for (int row = 0; row < table_info.numRows(); row++) {
				Integer[] cell_entities = table_info.getCellEntities(row, col);
				Object[] domain;
				double[] domain_prob;
				if (col == 5 && row == 1) {
					System.out.println(cell_entities.length);
				}
				if (cell_entities == null) {
					domain = new Object[] {unkent_id};
					domain_prob = new double[] {1.0};
				} else {
					domain = new Object[cell_entities.length + 1];
					domain_prob = new double[cell_entities.length + 1];
					double unk_prob = (1.0 / cell_entities.length) * new_entity_weight;
					double new_prob = (1.0 - unk_prob) / cell_entities.length;
					for (int ent = 0; ent < cell_entities.length; ent++) {
						Integer entity_id = cell_entities[ent];
						domain[ent] = entity_id;
						domain_prob[ent] = new_prob;
					}
					domain[cell_entities.length] = unkent_id;
					domain_prob[cell_entities.length] = unk_prob;
				}
				entity_variables[col][row] = createDiscreteVariable(domain, domain_prob,
																    ("Entity_" + row + "_" + col));
			}
		}
		
		if (max_columns != table_info.numVisibleCols()) {
			for (int row = 0; row < table_info.numRows(); row++) {
				HashSet<Integer> cvt_objects = table_info.retrieveAllValidCvtObjects(row);
				Iterator<Integer> it = cvt_objects.iterator();
				Object[] domain = new Object[cvt_objects.size() + 1];
				double[] domain_prob = new double[cvt_objects.size() + 1];
				double unk_prob = (1.0 / cvt_objects.size()) * new_entity_weight;
				if (cvt_objects.size() == 0) {
					unk_prob = 1.0;
				}
				double new_prob = (1.0 - unk_prob) / cvt_objects.size();
				for (int ent = 0; ent < cvt_objects.size(); ent++) {
					Integer entity_id = it.next();
					domain[ent] = entity_id;
					domain_prob[ent] = new_prob;
				}
				domain[cvt_objects.size()] = unkent_id;
				domain_prob[cvt_objects.size()] = unk_prob;
				for (int col = table_info.numVisibleCols(); col < max_columns; col++) {
					entity_variables[col][row] = createDiscreteVariable(domain, domain_prob,
							("Entity_" + row + "_" + col));
				}
			}
		}
	}
	
	private void initializePredicateAndColumnTypeVariables(HashSet<Integer> fixed_table_topic_ids, 
			ArrayList<HashMap<Integer, Integer> > root_predicates,
			HashSet<Integer> acceptable_preds) {
		int subjpred_id = table_info.getOrAddIdForString("SUBJPRED");
		int unkpred_id = table_info.getOrAddIdForString("UNKPRED");
		int unktype_id = table_info.getOrAddIdForString("UNKTYPE");
		HashSet<Integer> type_set = new HashSet<Integer>();
		possible_hidden_column_types_by_parent.clear();
		for (int col = 0; col < max_columns; col++) {
			HashMap<Integer, Integer> possible_predicate_ids = new HashMap<Integer, Integer>();
			for (int par = 0; par < max_columns; par++) {
				if (col == par) {
					possible_hidden_column_types_by_parent.add(new HashSet<Integer>());
					continue;
				}
				HashSet<Integer> hidden_column_types_for_parent = new HashSet<Integer>();
				acceptable_preds = col_to_acceptable_preds.get(new IntPair(par, col));
			
				if (col >= table_info.numVisibleCols() && par >= table_info.numVisibleCols()) {
					continue;
				}
				if (par < table_info.numVisibleCols() && col < table_info.numVisibleCols()) {
					HashMap<Integer, Integer> found_pred_ids = table_info.findAllPredicatesForDomainsAndRanges(possible_visible_column_types.get(par),
					                                                                                  possible_visible_column_types.get(col));
					if (acceptable_preds != null) {
						for (Integer pred_id : found_pred_ids.keySet()) {
							if (acceptable_preds.contains(pred_id)) {
								possible_predicate_ids.put(pred_id, found_pred_ids.get(pred_id));
								PredicateInfo p_info = table_info.getPredicateInfo(pred_id);
								type_set.add(p_info.getRangeId());
								hidden_column_types_for_parent.add(p_info.getRangeId());
							}
						}
					}
				} else if (par < table_info.numVisibleCols() && col >= table_info.numVisibleCols()) {
					HashMap<Integer, Integer> found_pred_ids = table_info.findAllPredicatesForDomainsAndRanges(possible_visible_column_types.get(par),
                            possible_hidden_column_types);
					if (acceptable_preds != null) {
						for (Integer pred_id : found_pred_ids.keySet()) {
							if (acceptable_preds.contains(pred_id)) {
								possible_predicate_ids.put(pred_id, found_pred_ids.get(pred_id));
								PredicateInfo p_info = table_info.getPredicateInfo(pred_id);
								type_set.add(p_info.getRangeId());
								hidden_column_types_for_parent.add(p_info.getRangeId());
							}
						}
					}
				} else {
					HashMap<Integer, Integer> found_pred_ids = table_info.findAllPredicatesForDomainsAndRanges(possible_hidden_column_types,
							possible_visible_column_types.get(col));
					if (acceptable_preds != null) {
						for (Integer pred_id : found_pred_ids.keySet()) {
							if (acceptable_preds.contains(pred_id)) {
								possible_predicate_ids.put(pred_id, found_pred_ids.get(pred_id));
								PredicateInfo p_info = table_info.getPredicateInfo(pred_id);
								type_set.add(p_info.getRangeId());
								hidden_column_types_for_parent.add(p_info.getRangeId());
							}
						}
					}
				}
				if (col == table_info.numVisibleCols()) {
					possible_hidden_column_types_by_parent.add(hidden_column_types_for_parent);
				}
			}
			
			
			if (ignore_table_topic) {
				possible_predicate_ids.put(subjpred_id, 1);
				if (col == table_info.numVisibleCols()) {
					possible_hidden_column_types_by_parent.add(new HashSet<Integer>());
				}
			} else {
				HashSet<Integer> hidden_column_types_for_parent = new HashSet<Integer>();
				if (root_predicates != null) {
					if (col > table_info.numVisibleCols()) {
						HashMap<Integer, Integer> preds_from_root = root_predicates.get(table_info.numVisibleCols());
						possible_predicate_ids.putAll(preds_from_root);
						for (Integer pred_id : preds_from_root.keySet()) {
							PredicateInfo p_info = table_info.getPredicateInfo(pred_id);
							type_set.add(p_info.getRangeId());
							hidden_column_types_for_parent.add(p_info.getRangeId());
						}
					} else {
						HashMap<Integer, Integer> preds_from_root = root_predicates.get(col);
						possible_predicate_ids.putAll(preds_from_root);
						for (Integer pred_id : preds_from_root.keySet()) {
							PredicateInfo p_info = table_info.getPredicateInfo(pred_id);
							type_set.add(p_info.getRangeId());
							hidden_column_types_for_parent.add(p_info.getRangeId());
						}
					}
				} else if (fixed_table_topic_ids != null) {
					// Must find the possible predicates.
					HashSet<Integer> root_type_ids = new HashSet<Integer>();
					for (Integer topic_id : fixed_table_topic_ids) {
						HashSet<Integer> type_ids = table_info.getTypesForEntityId(topic_id);
						if (type_ids != null) {
							root_type_ids.addAll(type_ids);
						}
					}
					HashMap<Integer, Integer> found_pred_ids = table_info.findAllPredicatesForDomainsAndRanges(root_type_ids,
                            possible_visible_column_types.get(col));
					if (acceptable_preds != null) {
						for (Integer pred_id : found_pred_ids.keySet()) {
							if (acceptable_preds.contains(pred_id)) {
								possible_predicate_ids.put(pred_id, found_pred_ids.get(pred_id));
								PredicateInfo p_info = table_info.getPredicateInfo(pred_id);
								type_set.add(p_info.getRangeId());
								hidden_column_types_for_parent.add(p_info.getRangeId());
							}
						}
					} else {
						possible_predicate_ids.putAll(found_pred_ids);
						for (Integer pred_id : found_pred_ids.keySet()) {
							PredicateInfo p_info = table_info.getPredicateInfo(pred_id);
							type_set.add(p_info.getRangeId());
							hidden_column_types_for_parent.add(p_info.getRangeId());
						}
					}
				}
				if (col == table_info.numVisibleCols()) {
					possible_hidden_column_types_by_parent.add(hidden_column_types_for_parent);
				}
			}

			Object[] type_arr;
			double[] type_prob_arr;
			Object[] domain_arr;
			double[] domain_prob;
			if (possible_predicate_ids.size() == 0) {
				domain_arr = new Integer[]{unkpred_id};
				domain_prob = new double[]{1.0};
				type_arr = new Integer[]{unktype_id};
				type_prob_arr = new double[]{1.0};
			} else {
				int full_cnt = 0;
				Collection<Integer> cnts = possible_predicate_ids.values();
				for (Integer cnt : cnts) {
					full_cnt += cnt.intValue();
				}
				full_cnt++;
				
				domain_arr = new Integer[possible_predicate_ids.size() + 1];
				domain_prob = new double[possible_predicate_ids.size() + 1];
				double unk_prob = (1.0 / full_cnt) * new_predicate_weight;
				double new_prob = (1.0 - unk_prob);
				if (possible_predicate_ids.size() == 0) {
					unk_prob = 1.0;
				}
				Iterator<Integer> it = possible_predicate_ids.keySet().iterator();
				for (int i = 0; i < possible_predicate_ids.size(); i++) {
					Integer pred_id = it.next();
					domain_arr[i] = pred_id;
					domain_prob[i] = new_prob / possible_predicate_ids.get(pred_id);
				}
				domain_arr[possible_predicate_ids.size()] = unkpred_id;
				domain_prob[possible_predicate_ids.size()] = unk_prob;
				
				type_arr = new Integer[type_set.size() + 1];
				type_prob_arr = new double[type_set.size() + 1];
				unk_prob = (1.0 / type_set.size()) * new_type_weight;
				new_prob = (1.0 - unk_prob) / type_set.size();
				if (type_set.size() == 0) {
					unk_prob = 1.0;
				}
				it = type_set.iterator();
				for (int i = 0; i < type_set.size(); i++) {
					Integer type_id = it.next();
					type_arr[i] = type_id;
					type_prob_arr[i] = new_prob;
				}
				type_arr[type_set.size()] = unktype_id;
				type_prob_arr[type_set.size()] = unk_prob;
			}
			column_type_variables[col] = createDiscreteVariable(type_arr, type_prob_arr, ("ColType" + col));
			predicate_variables[col] = createDiscreteVariable(domain_arr, domain_prob, ("Pred" + col));
		}
	}
	
	private Discrete createDiscreteVariable(Object[] domain, double[] prob, String label) {
		DiscreteDomain discrete_domain = DiscreteDomain.create(domain);
		Discrete var = new Discrete(discrete_domain);
		var.setInput(prob);
		var.setLabel(label);
		return var;
	}

	public double getApproximateTypeDomainSize() {
		return approximate_type_domain_size;
	}

	public void setApproximateTypeDomainSize(int approximate_type_domain_size) {
		this.approximate_type_domain_size = approximate_type_domain_size;
	}

	public double getProbOfWrongFunctionalRel() {
		return prob_of_wrong_functional_rel;
	}
	
	public double getProbOfNewObjectInNonFunctionalRel() {
		return prob_of_new_obj_in_nonfunc_rel;
	}

	public void setProbOfWrongFunctionalRel(double prob_of_wrong_functional_rel) {
		this.prob_of_wrong_functional_rel = prob_of_wrong_functional_rel;
	}

	public double getLaplaceSmoothingAlpha() {
		return laplace_smoothing_alpha;
	}

	public void setLaplaceSmoothingAlpha(double laplace_smoothing_alpha) {
		this.laplace_smoothing_alpha = laplace_smoothing_alpha;
	}
	
	private HashMap<Integer, HashSet<Integer> > collectTypesToEntitiesForCell(Object[] entities, 
			TableInformation table_info, int col) {
		HashMap<Integer, HashSet<Integer> > type_to_entity_idxs_map = new HashMap<Integer, HashSet<Integer> >();
		for (int ent = 0; ent < entities.length; ent++) {
			Integer entity_id = (Integer)(entities[ent]);
			HashSet<Integer> ent_types;
//			if (table_info.isLeafColumn(col)) {
//				ent_types = getPossibleTypesForColumn(col);
//			} else {
				ent_types = table_info.getTypesForEntityId(entity_id);
				if (ent_types == null)
					continue;
//			}
			for (Integer type : ent_types) {
				HashSet<Integer> entity_idxs = type_to_entity_idxs_map.get(type);
				if (entity_idxs == null) {
					entity_idxs = new HashSet<Integer>();
					type_to_entity_idxs_map.put(type, entity_idxs);
				}
				entity_idxs.add(ent);
			}
		}
		return type_to_entity_idxs_map;
	}
	
	public void createCellFactorCache() {
		if (ignore_table_topic) {
			entity_factor_cache = new FactorTable[max_columns][max_columns][table_info.numRows()];	
		} else {
			entity_factor_cache = new FactorTable[max_columns + 1][max_columns][table_info.numRows()];
		}
		
		HashMap<Pair<Integer, Integer>, HashMap<Integer, HashSet<Integer> > > type_to_entity_idxs_maps = 
				new HashMap<Pair<Integer, Integer>, HashMap<Integer, HashSet<Integer> > >();
		for (int row = 0; row < table_info.numRows(); row++) {
			for (int col = 0; col < max_columns; col++) {
				Object[] entities = entity_variables[col][row].getDomain().getElements();
				HashMap<Integer, HashSet<Integer> > type_to_entity_idxs_map = collectTypesToEntitiesForCell(entities, table_info, col);
				type_to_entity_idxs_maps.put(new Pair<Integer, Integer>(row, col), type_to_entity_idxs_map);
			}
		}
		
		for (int subj_col = 0; subj_col < max_columns; subj_col++) {
			if (table_info.isLeafColumn(subj_col))
				continue;
			for (int obj_col = 0; obj_col < max_columns; obj_col++) {
				if (subj_col == obj_col) {
					continue;
				}
				if (subj_col >= table_info.numVisibleCols() && obj_col >= table_info.numVisibleCols()) {
					continue;
				}
				for (int row = 0; row < table_info.numRows(); row++) {
					Pair<int[][], double[]> sparse_info = EntityFactor.createFactor(entity_variables[subj_col][row],
							predicate_variables[obj_col],
						   entity_variables[obj_col][row], this, obj_col,
						   type_to_entity_idxs_maps.get(new Pair<Integer, Integer>(row, obj_col)));
					entity_factor_cache[subj_col][obj_col][row] = FactorTable.create(sparse_info.getFirst(), sparse_info.getSecond(),
							entity_variables[subj_col][row].getDomain(), predicate_variables[obj_col].getDomain(),
							entity_variables[obj_col][row].getDomain());
				}
			}
		}
		
		if (!ignore_table_topic) {
			int subj_col = max_columns;
			for (int obj_col = 0; obj_col < max_columns; obj_col++) {
				if (table_info.isLeafColumn(obj_col)) {
					continue;
				}
				for (int row = 0; row < table_info.numRows(); row++) {
					Pair<int[][], double[]> sparse_info = EntityFactor.createFactor(topic_variables[obj_col],
							predicate_variables[obj_col],
						   entity_variables[obj_col][row], this, obj_col,
						   type_to_entity_idxs_maps.get(new Pair<Integer, Integer>(row, obj_col)));
					entity_factor_cache[subj_col][obj_col][row] = FactorTable.create(sparse_info.getFirst(), sparse_info.getSecond(),
							topic_variables[obj_col].getDomain(), predicate_variables[obj_col].getDomain(),
							entity_variables[obj_col][row].getDomain());
				}
			}
		}
	}
	
	public IFactorTable getEntityFactor(int subj_col, int obj_col, int row) {
		return entity_factor_cache[subj_col][obj_col][row];
	}
	
	public IFactorTable getPredicateFactor(int subj_col, int obj_col) {
		return predicate_factor_cache[subj_col][obj_col];
	}
	
	public IFactorTable getTopicFactor(int col) {
		return topic_factor_cache[col];
	}
	
	public void createTopicFactorCache() {
		if (ignore_table_topic) {
			return;
		}
		topic_factor_cache = new IFactorTable[max_columns];
		for (int col = 0; col < max_columns; col++) {
			if (table_info.isLeafColumn(col)) {
				continue;
			}
			Pair<int[][], double[]> sparse_info = RootFactor.createRootTopicFactor(topic_type_variables[col], topic_variables[col], this);
//			PrettyPrinter.prettyPrintPredicateFactor(sparse_info.getFirst(), sparse_info.getSecond(), topic_type_variables[col], topic_variables[col],
//					table_info);
			topic_factor_cache[col] = FactorTable.create(sparse_info.getFirst(), sparse_info.getSecond(),
					topic_type_variables[col].getDomain(), topic_variables[col].getDomain());
//			topic_factor_cache[col] = FactorTable.create(sparse_info.getFirst(), sparse_info.getSecond(),
//					topic_variables[col].getDomain(), topic_type_variables[col].getDomain());
		}
	}
	
	public void createPredicateFactorCache() {
		if (ignore_table_topic) {
			predicate_factor_cache = new FactorTable[max_columns][max_columns];
		} else {
			predicate_factor_cache = new FactorTable[max_columns + 1][max_columns]; 
		}
		
		for (int subj_col = 0; subj_col < max_columns; subj_col++) {
			if (table_info.isLeafColumn(subj_col)) {
				continue;
			}
			
			for (int obj_col = 0; obj_col < max_columns; obj_col++) {
				if (subj_col == obj_col) {
					continue;
				}
				if (subj_col >= table_info.numVisibleCols() && obj_col >= table_info.numVisibleCols()) {
					continue;
				}
				Pair<int[][], double[]> sparse_info = PredicateFactor.createFactor(column_type_variables[subj_col], predicate_variables[obj_col], this);
				predicate_factor_cache[subj_col][obj_col] = FactorTable.create(sparse_info.getFirst(), sparse_info.getSecond(),
						column_type_variables[subj_col].getDomain(), predicate_variables[obj_col].getDomain());
//				PredicateFactor.prettyPrintFactor(predicate_factor_cache[subj_col][obj_col], column_type_variables[subj_col],
//						predicate_variables[obj_col], table_info);
			}
		}
	
		if (!ignore_table_topic) {
			int subj_col = max_columns;
			for (int obj_col = 0; obj_col < max_columns; obj_col++) {
				if (table_info.isLeafColumn(obj_col)) {
					continue;  // Can't be a root column.
				}
				Pair<int[][], double[]> sparse_info = RootFactor.createRootPredicateFactor(topic_type_variables[obj_col], predicate_variables[obj_col], this);
				predicate_factor_cache[subj_col][obj_col] = FactorTable.create(sparse_info.getFirst(), sparse_info.getSecond(),
						topic_type_variables[obj_col].getDomain(), predicate_variables[obj_col].getDomain());
				if (obj_col == 1) {
					System.out.println("OBJ COL");
					PrettyPrinter.prettyPrintPredicateFactor(predicate_factor_cache[subj_col][obj_col], topic_type_variables[obj_col],
						predicate_variables[obj_col], table_info);
				}
			}
		}
	}
	
	
	public boolean isValidBackboneConnection(int subj_col, int obj_col) {
		return adj_col_pairs.size() == 0 ||
				adj_col_pairs.containsKey(new IntPair(subj_col, obj_col));
	}
	
	public void recreateHiddenBackboneAdjacencyMatrix() {
		int hidden_col_start = table_info.numVisibleCols();
		for (int obj_col = 0; obj_col < table_info.numVisibleCols(); obj_col++) {
			col_to_acceptable_preds.remove(new IntPair(hidden_col_start, obj_col));
			for (int ancestor_col = 0; ancestor_col < (hidden_col_start + 1); ancestor_col++) {
				if (obj_col == ancestor_col) {
					continue;
				}
				if (table_info.isLeafColumn(ancestor_col)) {
					continue;
				}
				HashSet<Integer> hidden_col_types = possible_hidden_column_types_by_parent.get(ancestor_col);
				for (int row = 0; row < table_info.numRows(); row++) {
					int found = 0;
					Object[] subj_entities = entity_variables[hidden_col_start][row].getDomain().getElements();
					if (subj_entities == null) {
						continue;
					}
					Object[] obj_entities = entity_variables[obj_col][row].getDomain().getElements();
					if (obj_entities == null) {
						continue;
					}
					for (int subj = 0; subj < subj_entities.length; subj++) {
						Integer subj_id = (Integer)(subj_entities[subj]);
						HashSet<Integer> subj_types = table_info.getTypesForEntityId(subj_id);
						if (subj_types == null) {
							continue;
						}
						boolean is_valid_type = false;
						for (Integer type : subj_types) {
							if (hidden_col_types.contains(type)) {
								is_valid_type = true;
								break;
							}
						}
						if (!is_valid_type) {
							continue;
						}
						for (int obj = 0; obj < obj_entities.length; obj++) {
							Integer obj_id = (Integer)(obj_entities[obj]);
							IntPair ent_pair = new IntPair(subj_id, obj_id);
							HashSet<Integer> pred_set = table_info.getPredsForSubjObjPair(ent_pair);
							
							if (pred_set != null) {
								IntPair col_pair = new IntPair((hidden_col_start + ancestor_col), obj_col);
								Integer cnt = adj_col_pairs.get(col_pair);

								if (found == 0) {
									if (cnt == null) {
										adj_col_pairs.put(col_pair, 1);
									} else {
										adj_col_pairs.put(col_pair, cnt.intValue() + 1);
									}
									found = 1;
									break;
								}
							}
						}
						if (found == 1) {
							break;
						}
					}
				}
			}
		}
	}
	
	public void createBackboneAdjacencyMatrix() {
		col_to_acceptable_preds = new HashMap<IntPair, HashSet<Integer> >(); 
		int max = table_info.numVisibleCols();
		if (max_columns != table_info.numVisibleCols()) {
			max++;
		}
		for (int subj_col = 0; subj_col < max; subj_col++) {
			if (subj_col < table_info.numVisibleCols() && table_info.isLeafColumn(subj_col)) {
				continue;
			}
			for (int obj_col = 0; obj_col < max; obj_col++) {
				if (subj_col == obj_col) {
					continue;
				}
				for (int row = 0; row < table_info.numRows(); row++) {
					int found = 0;
					Object[] subj_entities = entity_variables[subj_col][row].getDomain().getElements();
					if (subj_entities == null) {
						continue;
					}
					Object[] obj_entities = entity_variables[obj_col][row].getDomain().getElements();
					if (obj_entities == null) {
						continue;
					}
					for (int subj = 0; subj < subj_entities.length; subj++) {
						Integer subj_id = (Integer)(subj_entities[subj]);
						for (int obj = 0; obj < obj_entities.length; obj++) {
							Integer obj_id = (Integer)(obj_entities[obj]);
							IntPair ent_pair = new IntPair(subj_id, obj_id);
							HashSet<Integer> pred_set = table_info.getPredsForSubjObjPair(ent_pair);
							
							if (pred_set != null) {
								IntPair col_pair = new IntPair(subj_col, obj_col);
								Integer cnt = adj_col_pairs.get(col_pair);

								if (found == 0) {
									if (cnt == null) {
										adj_col_pairs.put(col_pair, 1);
									} else {
										adj_col_pairs.put(col_pair, cnt.intValue() + 1);
									}
									found = 1;
								}
								
								HashSet<Integer> acceptable = col_to_acceptable_preds.get(col_pair);
								if (acceptable == null) {
									acceptable = new HashSet<Integer>();
									col_to_acceptable_preds.put(col_pair, acceptable);
								}
								acceptable.addAll(pred_set);
							}
						}
					}
				}
				IntPair col_pair = new IntPair(subj_col, obj_col);
				Integer cnt = adj_col_pairs.get(col_pair);
				if (cnt == null) {
					adj_col_pairs.put(col_pair, 0);
				}
			}
		}
	}
	
	public List<Map.Entry<IntPair, Integer> > getSortedAdjColumnPairs() {
		List<Map.Entry<IntPair, Integer> > sorted_pairs = new ArrayList<Map.Entry<IntPair, Integer> >();
		sorted_pairs.addAll(adj_col_pairs.entrySet());
		Collections.sort(sorted_pairs, 
				new Comparator<Map.Entry<IntPair, Integer> >() {
			public int compare(Map.Entry<IntPair, Integer> e1, Map.Entry<IntPair, Integer> e2) {
				return e2.getValue().compareTo(e1.getValue());
			}
		});
		return sorted_pairs;
	}
}
